package com.chance.cc.crawler.development.bootstrap.dunkhome;

import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import groovy.util.logging.Slf4j;
import net.lingala.zip4j.core.ZipFile;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.script.crawler.CrawlerScript.$_type;

@Slf4j
public class DunkhomeHttpTest {
    public static Logger logger = LoggerFactory.getLogger(DunkhomeHttpTest.class);
    private static final String ITEM_VIDEO_URL = "http[s]*://video.sina.com.cn/[a-z0-9\\/]*/\\d{4}-\\d{2}-\\d{2}/[a-zA-Z]*-[a-zA-Z0-9]*.d.html";
    private static final String ITEM_NUM_URL = "https://\\d+.sina.com.cn/[a-z\\_\\/]*\\d{4}-\\d{2}-\\d{2}/[a-z]+-[a-z0-9]*.shtml";
    private static final String ITEM_AUTO_URL = "http[s]*://[a-z]+.sina.com.cn/[a-z\\/\\_0-9]*/\\d{4}-\\d{2}-\\d{2}[0-9\\/]*/[a-z]*-[a-z0-9]*.shtml\\S*";

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("H5168QRFNIU3804D");
        proxy.setPassword("5F6B3610BB719FAA");
    }

    public static void main(String[] args) {

        HttpConfig httpConfig = HttpConfig.me("get");
        httpConfig.setResponseTextGenerateHtml(false);
//        httpConfig.setCircularRedirectsAllowed(false);
//        httpConfig.setProxy(proxy);
//        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i=1;i<=1;i++){
            new Thread(new TestRun(downloader,httpConfig),"thread_"+i).start();
        }


//        String s  ="<div id=\"img_load_box\">\n" +
//                "<div class=\"container\" id=\"A\">\n" +
//                "</div>\n" +
//                "<div class=\"container\" id=\"B\">\n" +
//                "</div>\n" +
//                "</div>";
//        List<String> all = new Html(s).xpath("//div[@id=\"img_load_box\"]/div/@id").all();
//        System.out.println(all);

    }

    static class TestRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
//            String sourceUrl = "http://feed.mix.sina.com.cn/api/roll/get?pageid=13&lid=2503&num=50&page=%s";
//            String url = "http://www.ocn.com.cn/touzi/";
//            String url = "https://www.yicai.com/api/ajax/getjuhelist?action=news&page=2&pagesize=25";
//            String url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=13&lid=2503&num=50&page=1";
            String url = "https://news.yiche.com/info/categoryId0_p0_l0_f0_g0_c0_b0_3.html";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl(url);
//            httpRequest.addHeader("User-Agent","Mozilla/5.0 (Windows; U; Windows NT 6.0) AppleWebKit/534.24.7 (KHTML, like Gecko) Version/5.1 Safari/534.24.7");
//            httpRequest.setMethod(HttpConstant.Method.POST);
//            Map<String,Object> map = new HashMap<>();
//            map.put("type","news");
//            map.put("keywords","镁信");
//            httpRequest.setRequestBody(HttpRequestBody.form(map,"UTF-8"));
//            httpRequest.addHeader("referer","https://t.cj.sina.com.cn/article/search");
//            httpRequest.addHeader("content-type","application/x-www-form-urlencoded");
//            httpRequest.addHeader("Cookie","SUB=_2A25Nj-wiDeRhGeFL6lUV8CnIwzSIHXVu_VrqrDV8PUJbmtANLRLQkW9NQnCpH1UHoUwoSzLXMUcYnEyGXv9Eo_lw");
//            httpRequest.addHeader("User-Agent","Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36");
//            for(int i=0;i<400;i++){
//                httpRequest.setUrl(String.format(sourceUrl,i));
//                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
//                List<String> all = httpPage.getJson().jsonPath($_type + ".result.data").all();
//                List<String> allContent = new ArrayList<>();
//                for (String  data: all) {
//                    String string = JSONObject.parseObject(data).getString("url");
//                    if(!string.matches(ITEM_AUTO_URL) && !string.matches(ITEM_NUM_URL) && !string.matches(ITEM_VIDEO_URL) ){
//                        System.out.println(string);
//                    }
//                }

//                String minBehotTime = JSONObject.parseObject(allContent.get(0)).getJSONObject("info").getString("behot_time");
//                String maxBehotTime = JSONObject.parseObject(allContent.get(allContent.size() - 1)).getJSONObject("info").getString("behot_time");
//                String maxCursor = JSONObject.parseObject(allContent.get(allContent.size() - 1)).getJSONObject("info").getString("cursor");
//                url = sourceUrl.replace("#minBehotTime",minBehotTime).replace("#maxBehotTime",maxBehotTime).replace("#maxCursor",maxCursor);
//                for (String data : allContent) {
//                    JSONObject json = JSONObject.parseObject(data);
//                    String title = json.getJSONObject("info").getString("title");
//                    String type = json.getString("type");
//                    String articleUrl  = "https://www.dongchedi.com/ugc/article/" + json.getString("unique_id");
//                    System.out.println("title is ["+title+"] ，type is ["+type+"] ，article is ["+articleUrl+"]");
//                }
//            }
//            for (int i = 0; i < 20; i++) {
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                System.out.println(httpPage);
//            }
//            List<Selectable> nodes = httpPage.getHtml().xpath("//a").nodes();
//            for (Selectable node : nodes) {
//                String name = node.xpath("./font/text()").get();
//                if(StringUtils.isNotBlank(name) && name.equals("CSV")){
//                    System.out.println(node.xpath("./@href").get());
//                }
//            }

//            String fileParentPath ="F:\\chance_log\\zip\\";
//
//            try{
//                String filePath = fileParentPath + "";
//                IOUtils.write(httpPage.getBytes(),new FileOutputStream(filePath));
//                ZipFile zipFile = new ZipFile(filePath);
////                List<String> strings = IOUtils.readLines(new InputStreamReader(new FileInputStream(new File(filePath))));
////                httpPage.setJson(new Json(JSONObject.toJSONString(strings)));
//            } catch (Exception e) {
//                e.printStackTrace();
//            }


//            List<Selectable> nodes = httpPage.getHtml().xpath("//div[@id=\"img_load_box\"]/div").nodes();
//            List<String> m = new ArrayList<>();
//            for (Selectable node : nodes) {
//                List<String> n  = node.xpath(".//span/text()|.//p/text()").all();
//                m.addAll(n);
//            }
//            for (String s : m) {
//                System.out.println(s);
//            }
//
//            String s = httpPage.getJson().jsonPath($_type + ".pageIndex").get();
//            System.out.println(httpPage.getRawText());
        }
    }

    private static String getString(String regx, String input) {
        Pattern compile = Pattern.compile(regx);
        Matcher matcher = compile.matcher(input);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return null;
    }
}
